/*
*    Qizx/open 3.0 beta
*
* This code is the open-source version of Qizx.
* Copyright (C) 2004-2009 Axyana Software -- All rights reserved.
*
* The contents of this file are subject to the Mozilla Public License 
*  Version 1.1 (the "License"); you may not use this file except in 
*  compliance with the License. You may obtain a copy of the License at
*  http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
*  WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
*  for the specific language governing rights and limitations under the
*  License.
*
* The Initial Developer of the Original Code is Xavier Franc - Axyana Software.
*
*/
using System;
using System.Text;
using TextTokenizer = System.Qizx.Api.FullText.TextTokenizer;
using IntSet = System.Qizx.Util.Basic.IntSet;

namespace System.Qizx.Api.Util.FullText
{
	
	/// <summary> Generic Text Tokenizer, suitable for most Western languages.
	/// <p>Words are 1) a sequence of letters or digits, beginning with a letter; 
	/// 2) a number, without exponent. Words never contain a dash or an apostrophe.
	/// </summary>
	[Serializable]
	public class DefaultTextTokenizer : TextTokenizer
	{
		virtual public char[] TokenChars
		{
			get
			{
				int tlen = ptr - wordStart;
				if (tlen <= 0)
					return null;
				// dont remove words of length 1
				char[] word = new char[tlen];
				for (int iw = tlen; --iw >= 0; )
					word[iw] = txt[wordStart + iw];
				return word;
			}
			
		}
		virtual public int TokenOffset
		{
			get
			{
				return wordStart;
			}
			
		}
		virtual public int TokenLength
		{
			get
			{
				return ptr - wordStart;
			}
			
		}
		virtual public bool AcceptingWildcards
		{
			get
			{
				return acceptWildCards;
			}
			
			set
			{
				acceptWildCards = value;
			}
			
		}
		virtual public bool ParsingSpecialChars
		{
			get
			{
				return parseSpecialChars;
			}
			
			set
			{
				parseSpecialChars = value;
			}
			
		}
		[NonSerialized]
		private char[] txt;
		[NonSerialized]
		private int txtLength;
		[NonSerialized]
		private int ptr;
		[NonSerialized]
		private int wordStart;
		
		[NonSerialized]
		private bool parseSpecialChars;
		[NonSerialized]
		private bool acceptWildCards;
		
		[NonSerialized]
		private bool gotWildCard;
		[NonSerialized]
		private IntSet specialChars;
		
		
		public virtual void  start(char[] text, int length)
		{
			txt = text;
			txtLength = length;
			ptr = 0;
		}
		
		public virtual void  start(string text)
		{
			int len = text.Length;
			if (text is System.String)
			{
				this.txt = ((System.String) text).ToCharArray();
			}
			else
			{
				// copy!
				if (this.txt == null || this.txt.Length < len)
					this.txt = new char[len];
				for (int i = 0; i < len; i++)
				{
					this.txt[i] = text[i];
				}
			}
			this.txtLength = len;
			ptr = 0;
		}
		
		public virtual void  copyTokenTo(char[] array, int start)
		{
			int tlen = ptr - wordStart;
			for (int iw = tlen; --iw >= 0; )
				array[start + iw] = txt[wordStart + iw];
		}
		
		public virtual void  defineSpecialChar(char ch)
		{
			if (specialChars == null)
				specialChars = new IntSet(ch);
			else
				specialChars.add(ch);
		}
		
		public virtual bool gotWildcard()
		{
			return gotWildCard;
		}
		
		public virtual int nextToken()
		{
			gotWildCard = false;
			wordStart = ptr;
			for (; ptr < txtLength; ptr++)
			{
				wordStart = ptr;
				char ch = txt[ptr];
				if (System.Char.IsLetter(ch) || testWildcard(ch))
				{
					// simple way: break on dash and quote/apostrophe
					// accept dot always if acceptWildCards (query parsing)
					++ptr;
					for (; ptr < txtLength; ptr++)
					{
						ch = txt[ptr];
						if (ch == '\\' && acceptWildCards && ptr < txtLength)
						{
							++ptr;
						}
						else if (!System.Char.IsLetter(ch) && !System.Char.IsDigit(ch) && !testWildcard(ch))
							break;
					}
					return System.Qizx.Api.FullText.TextTokenizer_Fields.WORD;
				}
				else if (System.Char.IsDigit(ch))
				{
					// accept dot and comma inside number (but not at end)
					++ptr;
					for (; ptr < txtLength; ptr++)
					{
						ch = txt[ptr];
						if (!System.Char.IsDigit(ch) && !testWildcard(ch) && !((ch == '.' || ch == ',') && System.Char.IsDigit(charAhead(1))))
							break;
					}
					return System.Qizx.Api.FullText.TextTokenizer_Fields.WORD;
				}
				else if (parseSpecialChars && specialChars != null && specialChars.test(ch))
				{
					++ptr;
					return ch;
				}
				else if (ch == '\\' && acceptWildCards && ptr < txtLength)
				{
					++ptr;
				}
				else
				{
				} // throw away
			}
			wordStart = ptr;
			return System.Qizx.Api.FullText.TextTokenizer_Fields.END;
		}
		
		protected internal virtual bool testWildcard(char c)
		{
			if (!acceptWildCards)
				return false;
			if (c != '.')
				return false;
			char ch2 = charAhead(1);
			if (ch2 == '?' || ch2 == '+' || ch2 == '*')
				++ptr;
			else if (ch2 == '{')
			{
				int cp = ptr + 1;
				for (; cp < txtLength && txt[cp] != '}'; )
					++cp;
				if (cp < txtLength)
					++cp;
				ptr = cp;
			}
			gotWildCard = true;
			return true;
		}
		
		protected internal virtual char charAhead(int offset)
		{
			int pos = ptr + offset;
			return (pos < 0 || pos >= txtLength)?(char) 0:txt[pos];
		}
	}
}